{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# e) 行列重采样参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dpath = './data/'\n",
    "\n",
    "train = pd.read_csv(dpath+'RentListingInquries_FE_train.csv')\n",
    "# train = train.iloc[:100,:]\n",
    "y_train = train['interest_level']\n",
    "X_train = train.drop('interest_level',axis=1)\n",
    "X_train = np.array(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare cross validation \n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "kfold = list(kfold.split(X_train, y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.95], 'subsample': [0.6, 0.65, 0.7, 0.75, 0.8]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Best: -0.581127 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n",
    "# Best: -0.580125 using {'colsample_bytree': 0.95, 'subsample': 0.8}\n",
    "# 继续调整参数\n",
    "\n",
    "subsample = [i/20.0 for i in range(12,17)]\n",
    "colsample_bytree = [0.95]\n",
    "param_test5 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58282, std: 0.00485, params: {'colsample_bytree': 0.95, 'subsample': 0.6},\n",
       "  mean: -0.58221, std: 0.00353, params: {'colsample_bytree': 0.95, 'subsample': 0.65},\n",
       "  mean: -0.58118, std: 0.00353, params: {'colsample_bytree': 0.95, 'subsample': 0.7},\n",
       "  mean: -0.58092, std: 0.00344, params: {'colsample_bytree': 0.95, 'subsample': 0.75},\n",
       "  mean: -0.58012, std: 0.00350, params: {'colsample_bytree': 0.95, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.95, 'subsample': 0.8},\n",
       " -0.58012494559509542)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=181,  # 根据第4个文件，得到最优值是181\n",
    "        max_depth=7, # 根据上一轮结果，得到最优深度是7，weight是7\n",
    "        min_child_weight=7,\n",
    "        reg_alpha=1,\n",
    "        reg_lambda=0.75,# 根据上一轮结果，给出正则参数\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5.fit(X_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,gsearch5.best_score_\n",
    "# modelfit(xgb5, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 248.90564556,  243.75416751,  235.1693049 ,  232.21786971,\n",
       "         207.88291788]),\n",
       " 'mean_score_time': array([ 1.60607705,  0.98516126,  1.12527404,  0.93434262,  0.99838161]),\n",
       " 'mean_test_score': array([-0.5828232 , -0.58221161, -0.58117533, -0.58092273, -0.58012495]),\n",
       " 'mean_train_score': array([-0.44704254, -0.44525284, -0.44398304, -0.44288937, -0.44276077]),\n",
       " 'param_colsample_bytree': masked_array(data = [0.95 0.95 0.95 0.95 0.95],\n",
       "              mask = [False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_subsample': masked_array(data = [0.6 0.65 0.7 0.75 0.8],\n",
       "              mask = [False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'colsample_bytree': 0.95, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.65},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.75},\n",
       "  {'colsample_bytree': 0.95, 'subsample': 0.8}],\n",
       " 'rank_test_score': array([5, 4, 3, 2, 1], dtype=int32),\n",
       " 'split0_test_score': array([-0.57430668, -0.57643157, -0.57554758, -0.57551845, -0.57469158]),\n",
       " 'split0_train_score': array([-0.44780185, -0.44593109, -0.44412846, -0.44399795, -0.44436925]),\n",
       " 'split1_test_score': array([-0.58197598, -0.58060191, -0.57860803, -0.57897443, -0.5779568 ]),\n",
       " 'split1_train_score': array([-0.44716552, -0.44559259, -0.44466994, -0.44213057, -0.44311345]),\n",
       " 'split2_test_score': array([-0.58295324, -0.5823565 , -0.58281357, -0.58144383, -0.58047311]),\n",
       " 'split2_train_score': array([-0.44661598, -0.44500244, -0.44304814, -0.44103061, -0.44198605]),\n",
       " 'split3_test_score': array([-0.58833473, -0.58571044, -0.58439715, -0.58317367, -0.58326427]),\n",
       " 'split3_train_score': array([-0.44539542, -0.44378638, -0.44284387, -0.44398075, -0.44175037]),\n",
       " 'split4_test_score': array([-0.58654651, -0.58595877, -0.58451132, -0.58550467, -0.58424022]),\n",
       " 'split4_train_score': array([-0.44823393, -0.4459517 , -0.44522477, -0.44330697, -0.44258473]),\n",
       " 'std_fit_time': array([  0.5663118 ,   2.59997877,   0.42862087,   1.20010378,  36.39314607]),\n",
       " 'std_score_time': array([ 0.13333178,  0.28919071,  0.23651896,  0.35404631,  0.27712291]),\n",
       " 'std_test_score': array([ 0.00484786,  0.00352997,  0.00353428,  0.00344476,  0.00349665]),\n",
       " 'std_train_score': array([ 0.00099056,  0.00080957,  0.00091724,  0.00115111,  0.00093421])}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.580125 using {'colsample_bytree': 0.95, 'subsample': 0.8}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4VGX6//H3nQIRIZQEMBAwoSgg\nBJDQm8KCoIiCihSVoIhlEXVXf+qyruWrK9ZVQBcBFQQVBFcFUbCBCtIChBqkCwGkhd5J7t8f5yQO\nIWUSMpmU+3VdczFz5pxn7jMe55NTnueIqmKMMcbktwB/F2CMMaZ4soAxxhjjExYwxhhjfMICxhhj\njE9YwBhjjPEJCxhjjDE+YQFjjDHGJyxgjDHG+IQFjDHGGJ8I8ncB/hQeHq5RUVH+LsMYY4qUZcuW\n7VfVyjnNV6IDJioqivj4eH+XYYwxRYqI/O7NfHaIzBhjjE9YwBhjjPEJCxhjjDE+UaLPwRhTkM6e\nPUtSUhKnTp3ydynGeCUkJITIyEiCg4PztLwFjDEFJCkpiXLlyhEVFYWI+LscY7Klqhw4cICkpCSi\no6Pz1IYdIjOmgJw6dYqwsDALF1MkiAhhYWEXtcdtAWNMAbJwMUXJxW6vFjB5sHHPUd74bgMnz6T4\nuxRjjCm0LGDy4Mf1exn5w0b+8sZPfL16N6rq75KMMabQsYDJg/s61ubT+1oTekkwD360nAHjF7Nh\nz1F/l2VMtg4dOsQ777yTp2XffPNNTpw4kW+1TJgwgaFDh+Zbe96aN28ePXr0yPVyua1327ZtfPzx\nx7n+HG8lJyfTpUsX6tatS5cuXTh48GCm8z3xxBM0bNiQhg0bMnXq1PTpcXFxREdH06RJE5o0aUJC\nQoJP6rSAyaMW0ZWYObQt/3fTVazddYTub/3CczPXcvjkWX+XZkymClPAFHfZBcy5c+cuuv0RI0bQ\nuXNnNm7cSOfOnRkxYsQF88yaNYvly5eTkJDA4sWLefXVVzly5Ej6+6+++ioJCQkkJCTQpEmTi64p\nM3aZ8kUICgzgztZR9Iipxmvf/saEX7cxI2EXT3Srx63NIgkIsBO6JnPPzVzLul1Hcp4xFxpUC+WZ\nG6/K8v0nn3ySzZs306RJE7p06UKVKlX49NNPOX36NL169eK5557j+PHj9OnTh6SkJFJSUnj66afZ\ns2cPu3bt4tprryU8PJy5c+dm2v7s2bP5xz/+QUpKCuHh4fzwww8kJydz9913s2XLFsqUKcPYsWOJ\niYk5b7lp06bx3HPPERgYSPny5fn555/Ztm0bd955J8ePHwdg9OjRtGnThnnz5vHMM89QtWpVEhIS\n6N27N40aNeKtt97i5MmTfPHFF9SuXZu4uDhCQkJYu3Yte/bs4Y033rhgz+X48eM89NBDrF69mnPn\nzvHss89y0003Zfn97dixg27durF161b69+/PM888w9NPP014eDgPP/wwAMOHD6dq1ap8/PHHJCYm\n0qRJEwYOHEjFihWZNWsWp06d4vjx4/z444+8+uqrF3z/AJMnT2bkyJGcOXOGli1b8s477xAYGHhe\nLV9++SXz5s0DYODAgVxzzTW8/PLL582zbt06OnbsSFBQEEFBQTRu3JjZs2fTp0+fLNcxv9keTD6o\neGkpXuzViJlD2xEdfin/77NV9HpnAQk7Dvm7NGPSjRgxgtq1a5OQkECXLl3YuHEjS5YsISEhgWXL\nlvHzzz8ze/ZsqlWrxsqVK1mzZg3dunVj2LBhVKtWjblz52YZLvv27ePee+/ls88+Y+XKlUybNg2A\nZ555hqZNm7Jq1Sr+/e9/c9ddd12w7PPPP8+cOXNYuXIlM2bMAKBKlSp89913LF++nKlTpzJs2LD0\n+VeuXMlbb73F6tWrmTRpEhs2bGDJkiUMHjyYUaNGpc+3bds2fvrpJ2bNmsX9999/weW2L774Ip06\ndWLp0qXMnTuXxx9/PD3QMrNkyRI++ugjEhISmDZtGvHx8dxzzz1MnDgRgNTUVKZMmcKAAQMYMWIE\n7du3JyEhgUcffRSAhQsXMnHiRH788Ue+/fbbTL//xMREpk6dyoIFC0hISCAwMJCPPvoIgMGDB6cP\nzrtnzx4iIiIAiIiIYO/evRfU27hxY7755htOnDjB/v37mTt3Ljt27Eh/f/jw4cTExPDoo49y+vTp\nLNf7YtgeTD5qWL080+5vzRcJO3np6/Xc/PYCbmsWyf/rVo/K5Ur7uzxTiGS3p1EQvv32W7799lua\nNm0KwLFjx9i4cSPt27fnscce44knnqBHjx60b9/eq/YWLVpEhw4d0jvkVapUCYD58+fz2WefAdCp\nUycOHDjA4cOHz1u2bdu2xMXF0adPH3r37g04ox4MHTo0/Ud2w4YN6fM3b948/ce1du3adO3aFYBG\njRqdF4B9+vQhICCAunXrUqtWLdavX3/BdzBjxgxee+01wOmntH37durXr5/pOnbp0oWwsDAAevfu\nzfz583nkkUcICwtjxYoV7Nmzh6ZNm6bPk9nyad9LVt//qlWrWLZsGc2bNwfg5MmTVKlSBYDx48dn\n2m5WunbtytKlS2nTpg2VK1emdevWBAU5P/kvvfQSl112GWfOnGHIkCG8/PLL/Otf/8pV+96wgMln\nIkKvppF0aXAZo37cyPvztzJ7zR88/Je6DGwTRXCg7TQa/1NVnnrqKe67774L3lu2bBlff/01Tz31\nFF27dvXqh0dVM+0zkdkVlhnnGzNmDIsXL2bWrFnpJ5xHjRpF1apVWblyJampqYSEhKTPX7r0n3+s\nBQQEpL8OCAg47/xGxs/J+FpV+eyzz7jyyitzXL/s2hs8eDATJkzgjz/+4O67785y+UsvvfS8z87s\n+x81ahQDBw7kpZdeyraWqlWrsnv3biIiIti9e3d6CGU0fPhwhg8fDkD//v2pW7cuQHpAly5dmkGD\nBqWHbH6zXzsfKVs6iKe612f2Ix24+vKKvDArkevf+oUFm/b7uzRTQpUrV46jR52rHa+77jref/99\njh07BsDOnTvZu3cvu3btokyZMtxxxx089thjLF++/IJlM9O6dWt++ukntm7dCjhXOQF06NAh/RDP\nvHnzCA8PJzQ09LxlN2/eTMuWLXn++ecJDw9nx44dHD58mIiICAICApg0aRIpKbnvczZt2jRSU1PZ\nvHkzW7ZsuSBIrrvuOkaNGpUegitWrMi2ve+++47k5OT0cz1t27YFoFevXsyePZulS5dy3XXXATl/\nX1l9/507d2b69Onph7ySk5P5/fcLb73Ss2fP9ENzEydOzPTcUUpKCgcOHABg1apVrFq1Kn1vb/fu\n3YATdF988QUNGzbMdt3zyvZgfKx25bJMGNScHxL38vxX6xgwfjHdG17G8BvqE1mxjL/LMyVIWFgY\nbdu2pWHDhnTv3p3+/fvTunVrAMqWLcvkyZPZtGkTjz/+OAEBAQQHB/Pf//4XgCFDhtC9e3ciIiIy\nPQ9TuXJlxo4dS+/evUlNTU0/h/Lss88yaNAgYmJiKFOmTPqPoqfHH3+cjRs3oqp07tyZxo0b8+CD\nD3LLLbcwbdo0rr322vP++vfWlVdeSceOHdmzZw9jxow5by8I4Omnn+aRRx4hJiYGVSUqKoqvvvoq\ny/batWvHnXfeyaZNm+jfvz+xsbEAlCpVimuvvZYKFSqkn4yPiYlJP7EeFxdHxYoVz2ura9euJCYm\nXvD9N2jQgBdeeIGuXbuSmppKcHAwb7/9NpdffjmDBw/m/vvvJzY2lieffJI+ffrw3nvvUbNmzfRz\nXvHx8YwZM4bx48dz9uzZ9EOcoaGhTJ48Of0Q2YABA9i3bx+qSpMmTRgzZkyuv19vSEnuJBgbG6sF\neUfLU2dTGP/LFt6eu5lUVR64pjb3d6xNSHBgzgubIi8xMTHL4/smf8XFxdGjRw9uvfVWn39Wamoq\nV199NdOmTUs/BFWcZLbdisgyVY3NaVk7RFaAQoIDGdqpLj/8vSNdGlTlze+d0QBmr/nDRgMwpgha\nt24dderUoXPnzsUyXC6W7cEU4B5MRgs3H+C5mWtZ/8dR2tUJ59meDahTpZzf6jG+VVz2YFq2bHnB\nZa2TJk2iUaNGfqoo/8yZM4cnnnjivGnR0dF8/vnnfqrI/y5mD8YCxo8BA3AuJZXJi37nje82cOJM\nCnFtonj4L3UpF5K3G/yYwisxMZF69erZiMqmyFBV1q9fb4fIiqqgwADi2kYz97FruC02kvcWbOXa\n135iWvwOUlNLbvgXRyEhIRw4cMAOh5oiIe2GYxkvjsgN24Px8x5MRquSDvHMjLWs2H6IpjUr8FzP\nq4iJrODvskw+sFsmm6Imq1sm2yEyLxTGgAFITVU+X7GTl75Zz4Hjp+nTrAaPd7uS8LI2GoAxxv/s\nEFkRFhAg3NIskrmPdWRwu2g+W57Eta/N44MFWzmXkurv8owxxisWMIVYuZBght/QgNmPtKdJjQo8\nN3MdN4ycz6+bbTQAY0zhZwFTBNSpUo4P727Bu3c24/iZc/Qft5i/frScnYdO+rs0Y4zJkgVMESEi\nXHfVZXz/t448+pcr+D5xD51fn8eoHzZy6mzux2kyxhhf82nAiEg3EflNRDaJyJOZvB8nIvtEJMF9\nDPZ47xURWSsiiSIyUhxlRGSWiKx33xvhTVvFSUhwIA//xRkNoFO9Krz+3Qa6/Ocnvl1rowEYYwoX\nnwWMiAQCbwPdgQZAPxFpkMmsU1W1ifsY7y7bBmgLxAANgeZAR3f+11S1HtAUaCsi3bNrq7iKrFiG\ndwY046PBLQkJCmTIpGUM/GApm/cd83dpxhgD+HYPpgWwSVW3qOoZYAqQ9f1Iz6dACFAKKA0EA3tU\n9YSqzgVw21wOROZ75UVI2zrhfP1we/7VowErth/kuv/8zL+/TuToqbP+Ls0YU8L5MmCqAzs8Xie5\n0zK6RURWich0EakBoKoLgbnAbvcxR1UTPRcSkQrAjcAP2bVVEgQHBnB3O2c0gN5XV2fsz1vo9PpP\n/G95ko0GYIzxG18GTGYDLmX8tZsJRKlqDPA9MBFAROoA9XH2TqoDnUSkQ3rDIkHAJ8BIVd2SXVsX\nFCUyRETiRSR+3759eV65wii8bGleubUxX/y1LdUqXMLfPl3JrWN+Zc3OwzkvbIwx+cyXAZMEeO5F\nRAK7PGdQ1QOqmjYs6zigmfu8F7BIVY+p6jHgG6CVx6JjgY2q+qYXbZ1HVceqaqyqxlauXDmPq1a4\nNalRgc8faMMrt8awPfkEN46ez1P/W03y8TP+Ls0YU4L4MmCWAnVFJFpESgF9gRmeM4hIhMfLnkDa\nYbDtQEcRCRKRYJwT/InuMi8A5YFHvGyrRAoIEPrE1uDHx65hUJtoPo3fwbWvzePDhdtsNABjTIHw\nWcCo6jlgKDAH58f+U1VdKyLPi0hPd7Zh7uXGK4FhQJw7fTqwGVgNrARWqupMEYkEhuNclbY8w+XI\nWbVVooWGBPOvGxsw++H2NKweyr++XEuPUfNZtOWAv0szxhRzNthlIRzs0ldUldlr/uCFWYnsPHSS\nGxtX4x/X1yOi/CX+Ls0YU4TYYJfmAiJC90YRfP+3jjzcuS7frv2DTq/9xNtzN9loAMaYfGcBUwJd\nUiqQR7tcwfd/60iHK8J5dc5vXPfmz/yQuMffpRljihELmBKsRqUyvHtnLJPuaUFQgHDPxHgGfbCE\nLTYagDEmH1jAGNrXrczsRzrwzxvqs3TbQa5782dGfLOeY6fP+bs0Y0wRZgFjAGc0gMHta/HjYx25\nqUl1xvy0mc6vz+OLFTttEE1jTJ5YwJjzVCkXwmu3NeZ/D7ahamgIj0xN4LYxC200AGNMrlnAmExd\nXbMiXzzYlhG9G7Fl/3F6jp7P8M9Xc9BGAzDGeMkCxmQpIEDo26Imc/9+DXe1jmLK0h1c+/o8Ji36\nnRQbRNMYkwMLGJOj8mWCebbnVcwa1o56l5Xj6S/W0GPUfJZsTfZ3acaYQswCxnit3mWhfHJvK0b3\nb8qhE2fo8+5CHp6ygj8On/J3acaYQsgCxuSKiNAjpho//L0jD3Wqwzdr/qDT6/N4Z94mTp+z0QCM\nMX+ygDF5UqZUEH/veiXfP9qRtnXCeWX2b3R78xfmrt/r79KMMYWEBYy5KDXDyjDurlgmDGqOAIMm\nLOWeCUvZtv+4v0szxviZBYzJF9dcWYXZj3Tgqe71WLTlAF3/8zOvzF7PcRsNwJgSywLG5JtSQQHc\n17E2Pz52DT1iInhn3mY6v/4TXybYaADGlEQWMCbfVQ0N4Y3bmzD9/taElS3Fw1MS6Ddukd2y2ZgS\nxgLG+ExsVCVmDG3Hi70asmL7IfpbyBhToljAGJ8KDBAGtLyc9wY2Z+v+4/Qft4gDx077uyxjTAGw\ngDEFol3d8PSQGTB+sYWMMSWABYwpMO3qhvN+XNqejIWMMcWdBYwpUG3rOCHze7ITMvstZIwptixg\nTIFrWyec9wc6ITPAQsaYYssCxvhFG4+Q6T9ukYWMMcWQBYzxm7SQ2Z58wkLGmGLIAsb4VRv3nMz2\n5BP0G2shY0xxYgFj/K5NbSdkdhx0QmbfUQsZY4oDCxhTKLSpHc4HcS1IOniS/uMsZIwpDixgTKHR\nunYY78c1t5AxppjwacCISDcR+U1ENonIk5m8Hyci+0QkwX0M9njvFRFZKyKJIjJSHGVEZJaIrHff\nG+Exf2kRmep+1mIRifLluhnfaF07jA8GOSHTb9wi9h612zEbU1T5LGBEJBB4G+gONAD6iUiDTGad\nqqpN3Md4d9k2QFsgBmgINAc6uvO/pqr1gKZAWxHp7k6/BzioqnWA/wAv+2jVjI+1quWEzM6DJ+k/\nbrGFjDFFlC/3YFoAm1R1i6qeAaYAN3m5rAIhQCmgNBAM7FHVE6o6F8BtczkQ6S5zEzDRfT4d6Cwi\nki9rYgqcZ8j0G2t7MsYURb4MmOrADo/XSe60jG4RkVUiMl1EagCo6kJgLrDbfcxR1UTPhUSkAnAj\n8EPGz1PVc8BhICz/VscUtFa1wpgwqDm7D5+ykDGmCPJlwGS295DxtoYzgShVjQG+x90DEZE6QH2c\nvZPqQCcR6ZDesEgQ8AkwUlW35OLzEJEhIhIvIvH79u3L5SqZgtayVhgfxHmEzBELGWOKCl8GTBJQ\nw+N1JLDLcwZVPaCqaZcKjQOauc97AYtU9ZiqHgO+AVp5LDoW2Kiqb2b2eW4AlQeSMxalqmNVNVZV\nYytXrpznlTMFp2WtMCYMauGEzDgLGWOKCl8GzFKgrohEi0gpoC8ww3MGEYnweNkTSDsMth3oKCJB\nIhKMc4I/0V3mBZzweCTD580ABrrPbwV+VLsRfLHRIrpSesj0tZAxpkjwWcC450GGAnNwwuFTVV0r\nIs+LSE93tmHu5cYrgWFAnDt9OrAZWA2sBFaq6kwRiQSG41yVtjzDpc3vAWEisgn4G3DBZdGmaEsL\nmT8sZIwpEqQk/5EfGxur8fHx/i7D5NLSbcnEvb+EqqEhTBnSiiqhIf4uyZgSRUSWqWpsTvNZT35T\n5DSPqsSEu1uw58gp+o5dxB7bkzGmULKAMUVS86hKTHRDpp+FjDGFkgWMKbJiLWSMKdQsYEyR5hky\nfccu4o/DFjLGFBYWMKbIi42qxIf3tGDf0dP0G2chY0xhYQFjioVml1di4t3NLWSMKUQsYEyx4YSM\nsyfTd+xCCxlj/MwCxhQrzS6vyMS7W7D/2Bn6jl3I7sMn/V2SMSWWBYwpds4PmUUWMsb4iQWMKZaa\nXV6RD+9pQbKFjDF+YwFjiq2ra1ZkokfI7DpkIWNMQbKAMcXa1TX/3JPpN85CxpiCZAFjir2mNc8/\nXGYhY0zByDFgRKS2iJR2n18jIsPc2xUbU2SkhczB407I7LSQMcbnvNmD+QxIcW9j/B4QDXzs06qM\n8YGmNSsyaXBLDh4/Qz8LGWN8zpuASXVvHtYLeFNVHwUicljGmEKpSY0KTsiccPrJWMgY4zveBMxZ\nEemHczvir9xpwb4ryRjfalKjApPvacmhE2ctZIzxIW8CZhDQGnhRVbeKSDQw2bdlGeNbjTOETNLB\nE/4uyZhiJ8eAUdV1qjpMVT8RkYpAOVUdUQC1GeNT54fMIgsZY/KZN1eRzRORUBGpBKwEPhCRN3xf\nmjG+17hGBT4a3JLDJy1kjMlv3hwiK6+qR4DewAeq2gz4i2/LMqbgxEQ6IXPEDZkdyRYyxuQHbwIm\nSEQigD78eZLfmGLFCZlWHDl5ln7jLGSMyQ/eBMzzwBxgs6ouFZFawEbflmVMwWsUWT49ZGxPxpiL\n581J/mmqGqOqD7ivt6jqLb4vzZiClxYyx06fs5Ax5iJ5c5I/UkQ+F5G9IrJHRD4TkciCKM4Yf3BC\npqWFjDEXyZtDZB8AM4BqQHVgpjvNmGKrYXULGWMuljcBU1lVP1DVc+5jAlDZx3UZ43cWMsZcHG8C\nZr+I3CEige7jDuCArwszpjDIGDLbD1jIGOMtbwLmbpxLlP8AdgO34gwfkyMR6SYiv4nIJhF5MpP3\n40Rkn4gkuI/BHu+9IiJrRSRRREaKiLjTXxSRHSJyzNu2jLkYaSFz/Mw5+o2zkDHGW95cRbZdVXuq\namVVraKqN+N0usyWiAQCbwPdgQZAPxFpkMmsU1W1ifsY7y7bBmgLxAANgeZAR3f+mUCLLD72graM\nyQ8Nq5dn8j1OyPQdu9BCxhgv5PWOln/zYp4WwCb3suYzwBTgJi/bVyAEKAWUxhm9eQ+Aqi5S1d25\nL9mYi5O2J3PibAp9xy7k9wPH/V2SMYVaXgNGvJinOrDD43WSOy2jW0RklYhMF5EaAKq6EJiLc0hu\nNzBHVRO9+MwL2rqgcJEhIhIvIvH79u3zoklj/nRVtT9Dpt/YRRYyxmQjrwGjXsyTWQhlXG4mEKWq\nMcD3wEQA9+6Z9YFInFDqJCIdcvi8TNu6oADVsaoaq6qxlSvbxXAm966qVp6PB7dy92QsZIzJSpYB\nIyJHReRIJo+jOH1icpIEeO5FRAK7PGdQ1QOqetp9OQ5o5j7vBSxS1WOqegz4BmiV3Ydl05Yx+a5B\ntVA+HtyKUxYyxmQpy4BR1XKqGprJo5yqBnnR9lKgrohEi0gpoC9Oh8107iCaaXoCaYfBtgMdRSRI\nRIJxTvBne4gsm7aM8YkG1UL5yCNktu23kDHGU14PkeVIVc8BQ3EGykwEPlXVtSLyvIj0dGcb5l6K\nvBIYBsS506cDm4HVOPegWamqMyH98uUkoIyIJInIszm0ZYzPWMgYkzVR9eZ0SvEUGxur8fHx/i7D\nFAOJu48wYPxiSgUGMGVIK6LCL/V3Scb4jIgsU9XYnObz2R6MMSVJ/YhQPhrckjMpqfQdu4ittidj\njAWMMfmlfkQoH9/rhEw/CxljvBquP7OryXa4Q/jXKogijSkq6l32Z8j0HbvQQsaUaN7swbwBPI7T\nHyUSeAznMuApwPu+K82YoiktZM6mKH3HLmTLvmM5L2RMMeRNwHRT1XdV9aiqHlHVscD1qjoVqOjj\n+owpkupdFson97bibIrSb9wiCxlTInkTMKki0kdEAtxHH4/3Su4laMbk4MrLyvHJva04l6L0HWsh\nY0oebwJmAHAnsNd93AncISKX4PRzMcZk4crLyvHxva1ISbWQMSWPN8P1b1HVG1U13H3cqKqbVPWk\nqs4viCKNKcquvKwcnwz5M2Q2W8iYEsKbq8gi3SvG9orIHhH5TEQiC6I4Y4qLK6o6IZOqSj8LGVNC\neHOI7AOcMcSq4VxJNtOdZozJhSuqOofLUtX2ZEzJ4E3AVFbVD1T1nPuYANg498bkwRVVnRP/6obM\npr0WMqb48iZg9ovIHSIS6D7uAA74ujBjiqu66SED/cZZyJjiy5uAuRvoA/yBc3fJW4FBvizKmOLO\nCZmWFjKmWPPmKrLtqtpTVSurahVVvRnoXQC1GVOs1a1ajilDnJBxDpcd9XdJxuSrvA52+bd8rcKY\nEqpOFSdkAPqOXWwhY4qVvAaM5GsVxpRgGUNm4x4LGVM85DVgbIgYY/KREzKtEIF+4yxkTPGQZcBk\nMUz/ERE5itMnxhiTj+pUKcsn9zohc/vYRTw/cx2z1/xB8vEz/i7NmDyxWybbLZNNIbN53zGe+XIt\nS7clc/pcKgB1q5SlZa1KtIgOo2V0JaqGhvi5SlOSeXvLZAsYCxhTSJ0+l8LqpMMs3prMkq3JxG9L\n5viZFACiwsrQIvrPwImseAkidmrUFAwLGC9YwJii5FxKKut2H2HJ1mQWbUlm6bZkDp88C0C18iHp\ngdMiuhK1K19qgWN8xgLGCxYwpihLTVU27D3Kkq3JLN6SzOKtyew/dhqA8LKlnMCJckKn3mXlCAiw\nwDH5wwLGCxYwpjhRVbbuP+4EztZkFm85wK7DpwAIDQly93CcwGlYLZSgwLxeRGpKOm8DJqggijHG\n+J6IUKtyWWpVLkvfFjUBSDp4In0PZ8m2ZL5P3AtAmVKBNLu8Ii3dwGlcozylgwL9Wb4phixgjCnG\nIiuWIbJiGXpf7dzCae+RUyzZ5gbO1mRe+3YDAKWCAmhaowItoyvRslYYTWtWoEwp+3kwF8cOkdkh\nMlOCHTx+hqXbktOvVFu76zCpCkEBQqPI8rSIrkSr6DCaRVUkNCTY3+WaQsLOwXjBAsaY8x09dZb4\n3w+yxA2cVUmHOJuiBAjUjwilRXQlWrpXqlW6tJS/yzV+UigCRkS6AW8BgcB4VR2R4f044FVgpztp\ntKqOd997BbgBZ7SB74CHVVVF5EXgLqCiqpb1aKs08CHQDOd+Nber6rbs6rOAMSZ7J8+ksGL7wfQ9\nnOXbD57X+bOFe0jNOn+WLH4/yS8igcDbQBcgCVgqIjNUdV2GWaeq6tAMy7YB2gIx7qT5QEdgHs4t\nm0cDGzO0cw9wUFXriEhf4GXg9vxbI2NKnktKBdKmTjht6oQDF3b+/GLFTj5avB2Ay8PKpF80YJ0/\nDfj2JH8LYJOqbgEQkSnATUDGgMmMAiFAKZyRm4OBPQCqushtL+MyNwHPus+nA6NFRLQkHwM0Jp+V\nDgokNqoSsVGV+Ou153f+XLw1mW/X7eHT+CQAIsqHpAeOdf4smXwZMNWBHR6vk4CWmcx3i4h0ADYA\nj6rqDlVdKCJzce6gKTiHzhJsDuSlAAAUq0lEQVS9/TxVPScih4EwYP9FrocxJgtBgQHERFYgJrIC\ng9vXOr/z59Zk5m86wBcJuwDr/FkS+TJgMttyMu5NzAQ+UdXTInI/MBHoJCJ1gPpApDvfdyLSQVV/\nvsjPQ0SGAEMAatasmcMqGGNyIyBAqHdZKPUuC+Wu1lHndf5MC52vV/8BXNj586pqoQRb589ixZcB\nkwTU8HgdCezynEFVD3i8HIdz3gSgF7BIVY8BiMg3QCsgu4BJ+7wkEQkCygPJGWdS1bHAWHBO8udi\nfYwxuZRd58+0wLHOn8WXLwNmKVBXRKJxrhLrC/T3nEFEIlR1t/uyJ5B2GGw7cK+IvISzZ9IReDOH\nz5sBDAQWArcCP9r5F2MKn6w6f6aNOJBZ588W0WFcfbl1/ixqfH2Z8vU4wRAIvK+qL4rI80C8qs5w\nA6QncA5nb+MBVV3vXoH2DtAB5zDXbFX9m9vmKzhBVQ1nj2i8qj4rIiHAJKCp21bftAsMsmKXKRtT\n+KR1/kzbw8ms82fLaOdCA+v86R+Foh9MYWcBY0zhd/TUWZa5nT8Xe3T+FIEGbufPVrXC6Fyvig3g\nWUAsYLxgAWNM0XPyTAordhxMP6SW1vmzVa1KjOzXlCrlrMOnr1nAeMECxpii78y5VL5YsZN/zVhD\naEgwo/o1pWWtMH+XVax5GzC2P2mMKdJKBQXQp3kNPn+wLZeWDqL/+MW8+9NmSvIfz4WFBYwxplio\nHxHKl0Pb0rVBVV76Zj33TVqWfktp4x8WMMaYYiM0JJh3BlzNP2+oz4/r99Jz9HzW7jrs77JKLAsY\nY0yxIiIMbl+LKUNacepsCr3e+ZWpS7f7u6wSyQLGGFMsxUZVYtaw9jSPqsgTn63m8WkrOXU2xd9l\nlSgWMMaYYiu8bGk+vLslD3Wqw7RlSfR651e27T/u77JKDAsYY0yxFhgg/L3rlXwQ15xdh05y46j5\nzF7zh7/LKhEsYIwxJcK19arw1UPtiK58KfdPXsa/v07kbEqqv8sq1ixgjDElRo1KZZh2f2vuaFWT\nsT9vYcC4xew5csrfZRVbFjDGmBKldFAgL9zciDdvb8LqnYe5YeR8ft1s9yX0BQsYY0yJdHPT6nw5\ntC2hlwRxx/jFvD13E6mp1vs/P1nAGGNKrCuqlmPG0HZ0bxTBq3N+494P4zl8wnr/5xcLGGNMiVa2\ndBCj+zXl2Rsb8PPGffQY/Qtrdlrv//xgAWOMKfFEhLi20Uy9rzXnUpTe//2VjxdvtwEzL5IFjDHG\nuK6uWZGvHmpHy+hK/OPz1fz905WcPGO9//PKAsYYYzyElS3NhEEteLhzXT5P2MnNby9gy75j/i6r\nSLKAMcaYDAIDhEe7XMGEQS3Ye/QUPUcv4OvVu/1dVpFjAWOMMVnoeEVlvhrWnjpVyvLgR8v5v6/W\nWe//XLCAMcaYbFSvcAmf3teauDZRvDd/K33HLmL34ZP+LqtIsIAxxpgclAoK4NmeVzGyX1MSdx+h\nx8j5zN9ovf9zYgFjjDFe6tm4GjOGtqXipaW48/3FjPpho/X+z4YFjDHG5EKdKuX48q9t6dm4Gq9/\nt4G7Jy7l4PEz/i6rULKAMcaYXLq0dBBv3t6E/7u5Ib9uOkCPUfNZueOQv8sqdCxgjDEmD0SEO1td\nzrT7WwNw25iFTFr0u/X+92ABY4wxF6FxjQp89VA72tQJ4+kv1vDo1ASOnz7n77IKBQsYY4y5SBUv\nLcX7A5vz9y5X8OXKXdz89gI27T3q77L8zqcBIyLdROQ3EdkkIk9m8n6ciOwTkQT3MdjjvVdEZK2I\nJIrISBERd3ozEVnttuk5/VkR2enR1vW+XDdjjPEUECA81Lkuk+5uSfLxM/QcvYCZK3f5uyy/8lnA\niEgg8DbQHWgA9BORBpnMOlVVm7iP8e6ybYC2QAzQEGgOdHTn/y8wBKjrPrp5tPUfj7a+9sV6GWNM\ndtrVDeerYe2oHxHKQ5+s4NkZazlzrmT2/vflHkwLYJOqblHVM8AU4CYvl1UgBCgFlAaCgT0iEgGE\nqupCdc6kfQjcnP+lG2NM3kWUv4QpQ1pxT7toJvy6jT7vLmTnoZLX+9+XAVMd2OHxOsmdltEtIrJK\nRKaLSA0AVV0IzAV2u485qproLp+UTZtD3bbeF5GKmRUlIkNEJF5E4vft25fnlTPGmOwEBwbwdI8G\nvDPgajbtPUaPkb/w04aS9Zvjy4CRTKZlvH5vJhClqjHA98BEABGpA9QHInECpJOIdMihzf8CtYEm\nOKH0emZFqepYVY1V1djKlSvnbo2MMSaXrm8UwYyhbalSLoS4D5bwn+82kFJCev/7MmCSgBoeryOB\n8854qeoBVT3tvhwHNHOf9wIWqeoxVT0GfAO0ctuMzKxNVd2jqimqmuq21SKf18cYY/KkVuWyfP7X\nNvRqUp23fthI3AdLSC4Bvf99GTBLgboiEi0ipYC+wAzPGdxzKml6Aonu8+1ARxEJEpFgnBP8iaq6\nGzgqIq3cq8fuAr7MpK1ewBpfrJQxxuRFmVJBvN6nMf/u1YjFW5K5YeQvLN9+0N9l+ZTPAkZVzwFD\ngTk4wfGpqq4VkedFpKc72zD3UuSVwDAgzp0+HdgMrAZWAitVdab73gPAeGCTO8837vRX3MuXVwHX\nAo/6at2MMSYvRIT+LWvy2QNtCAwQbn93IRMWbC22vf+luK6YN2JjYzU+Pt7fZRhjSqDDJ87yt08T\n+GH9XnrERDDilhjKlg7yd1leEZFlqhqb03zWk98YY/ygfJlgxt0Vy//rdiVfr95Nz9Hz2bCnePX+\nt4Axxhg/CQgQHrymDpMHt+TIybPcNHoBX6zY6e+y8o0FjDHG+Fmb2uHMGtaeRtXL88jUBP75xWpO\nn0vxd1kXzQLGGGMKgaqhIXx0b0uGdKjF5EXbuW3MQnYkn/B3WRfFAsYYYwqJ4MAA/nF9fcbc0Yyt\n+47TY9R85q7f6++y8swCxhhjCpluDS9j5kPtqFbhEgZNWMprc34rkr3/LWCMMaYQigq/lM8fbMNt\nzSIZPXcTd72/mP3HTue8YCFiAWOMMYVUSHAgr97WmFduiSF+20FuGPkL8duS/V2W1yxgjDGmkOvT\nvAb/e7ANIcGB9B27iPG/bCkSvf8tYIwxpgi4qlp5ZgxtR6d6VXhhViIPfrSco6fO+rusbFnAGGNM\nEVH+kmDevbMZT3Wvx7fr9tBz9ALW/3HE32VlyQLGGGOKEBHhvo61+XhwS46dPsfNby9g+rKknBf0\nAwsYY4wpglrWCmPWsHY0qVGBx6at5Kn/reLU2cLV+98Cxhhjiqgq5UKYfE9LHrimNp8s2cGtY34t\nVL3/LWCMMaYICwoM4Ilu9Rh/VyzbD5zghpG/8P26Pf4uC7CAMcaYYuEvDary1UPtqRlWhsEfxvPy\n7PWcS0n1a00WMMYYU0zUDCvD9Pvb0K9FDf47bzN3vLeYvUdP+a0eCxhjjClGQoIDeal3DK/d1piE\nHYfoMXI+i7cc8EstFjDGGFMM3dosks8fbMulpYPoP34x7/60ucB7/1vAGGNMMVU/IpQZQ9vStUFV\nXvpmPfdNWsbhkwXX+98CxhhjirFyIcG8M+Bq/nlDfX5cv5eeo+ezdtfhAvlsCxhjjCnmRITB7Wsx\nZUgrTp1Nofc7vzJr1W6ff64FjDHGlBCxUZWYNaw97eqEExVexuefF+TzTzDGGFNohJctzXtxzQvk\ns2wPxhhjjE9YwBhjjPEJCxhjjDE+4dOAEZFuIvKbiGwSkSczeT9ORPaJSIL7GOzx3isislZEEkVk\npIiIO72ZiKx22/ScXklEvhORje6/FX25bsYYY7Lns4ARkUDgbaA70ADoJyINMpl1qqo2cR/j3WXb\nAG2BGKAh0Bzo6M7/X2AIUNd9dHOnPwn8oKp1gR/c18YYY/zEl3swLYBNqrpFVc8AU4CbvFxWgRCg\nFFAaCAb2iEgEEKqqC9UZ8+BD4GZ3mZuAie7ziR7TjTHG+IEvA6Y6sMPjdZI7LaNbRGSViEwXkRoA\nqroQmAvsdh9zVDXRXd7z3qCebVZV1d3u8ruBKvm5MsYYY3LHlwEjmUzLONLaTCBKVWOA73H3QESk\nDlAfiMQJkE4i0sHLNrMvSmSIiMSLSPy+fftys6gxxphc8GVHyySghsfrSGCX5wyq6jmG9DjgZfd5\nL2CRqh4DEJFvgFbAJLedzNrcIyIRqrrbPZS2N7OiVHUsMNZtd5+I/J6HdQMIB/bncVlfsrpyx+rK\nvcJam9WVOxdT1+XezOTLgFkK1BWRaGAn0Bfo7zlDWiC4L3sCie7z7cC9IvISzl5LR+BNNzyOikgr\nYDFwFzDKXWYGMBAY4f77ZU4FqmrlvK6ciMSramxel/cVqyt3rK7cK6y1WV25UxB1+SxgVPWciAwF\n5gCBwPuqulZEngfiVXUGMExEegLngGQgzl18OtAJWI1zCGy2qs5033sAmABcAnzjPsAJlk9F5B6c\ngLrNV+tmjDEmZz4di0xVvwa+zjDtXx7PnwKeymS5FOC+LNqMx7l0OeP0A0DniyzZGGNMPrGe/Hk3\n1t8FZMHqyh2rK/cKa21WV+74vC4p6FtoGmOMKRlsD8YYY4xPWMC4cho3zZ2nj4isc8dI+9hj+kB3\nDLSNIjLQY3qm46YVRF0i0kREFrrTVonI7R7zTxCRrR5jwDUpqLrc6Skenz3DY3q0iCx2v8epIlKq\noOoSkWs9akoQkVMicrP7ns+/LxH5j0f7G0TkkMd7ftu+sqrL39tXDt+X37avbL4vf29fNUVkrois\ncP97Xe/x3lPucr+JyHXetukVVS3xD5yr3DYDtXCGp1kJNMgwT11gBVDRfV3F/bcSsMX9t6L7PG2e\nJUBrnEutvwG6F2BdVwB13efVcEZEqOC+ngDc6o/vy31+LIt2PwX6us/HAA8UZF0e81TCuaqxTEF9\nXxnmfwjnqku/b1/Z1OXX7Suruvy9fWVXlz+3L5zzLQ+4zxsA2zyer8QZkivabScwt+ua1cP2YBze\njJt2L/C2qh4EUNW0jpzXAd+parL73ndAN8l+3DSf16WqG1R1o/t8F07H0zz3+8mvurLi/vXdCecS\ndcjbeHL5VdetwDeqeiKXn38xdXnqB3ziPvf39pVpXYVg+8q0rqwU4PblTV3+2L4UCHWfl+fPDuo3\nAVNU9bSqbgU2ue1dzFiS6SxgHN6Mm3YFcIWILBCRRSLSLYdlsxs3rSDqSiciLXD+CtnsMflFd1f5\nPyJSuoDrChFnuJ5FaYcJgDDgkKqey6ZNX9eVpi8X/jD4+vsCQEQux/lL8sccli2o7Surujzf88f2\nlV1d/ty+sqsrjT+2r2eBO0QkCafryEM5LOv1umbHAsbhzRhnQTiHV67B+ctkvIhUyGbZix437SLr\nchpw/tKdBAxS1VR38lNAPZzbIFQCnijgumqq04O4P/CmiNT2sk1f15X2fTXC6SCcpiC+rzR9genq\n9AXLbtmC+r6yqstpwH/bV3Z1+XP7yq4uf25f/YAJqhoJXA9MEpGAbJbNj+/LAsaV47hp7jxfqupZ\nd1fyN5wfqqyWTSLrcdMKoi5EJBSYBfxTVRelLaCqu9VxGvgAZ3e4wOpyD6mgqluAeUBTnDGRKohI\nUDZt+rQuVx/gc1U9mzahgL6vNBn/uvX39pVVXf7evrKsy8/bV5Z1ufy1fd2Dcw4KdUarD8EZiyy7\n7cvbdc1abk/aFMcHzl+1W3B2adNOaF2VYZ5uwET3eTjO7mMYzl8cW3FOwFZ0n1dy51uKM0hn2knY\n6wuwrlI4N157JJN2I9x/BXgTGFGAdVUESntM34h78hCYxvknYR8sqLo83l8EXFvQ35c735XANtz+\nae40v25f2dTl1+0rm7r8un1lVZe/ty93+4hzn9fHCQsBruL8k/xbcE7we7WuOdaW2wWK6wNnt3ED\nznHk4e6054GeHv/x3wDW4YyR1tdj2btxTo5twjlUkDY9Fljjtjk6sw3OV3UBdwBngQSPRxP3vR/d\nedcAk4GyBVhXG/f1SvffezzarIVzZdQm98egdAH/d4zCGZg1IEObPv++3NfPksmPiz+3r6zq8vf2\nlU1dft2+cvjv6LftC+dqsQXu95IAdPVYdri73G94XImYWZu5fVhPfmOMMT5h52CMMcb4hAWMMcYY\nn7CAMcYY4xMWMMYYY3zCAsYYY4xPWMAY4yMi8qyIPFYI6tgmIuH+rsOUPBYwxhhjfMICxphcEJFL\nRWSWiKwUkTUicrvnHoKIxIrIPI9FGovIj+49SO5154kQkZ/de3+sEZH27vT/uoM0rhWR5zw+c5uI\n/Fuc+6/Ei8jVIjJHRDaLyP3uPNe4bX4uzr1uxrhjTWWs/w4RWeJ+9rsiEujL78uUbBYwxuRON2CX\nqjZW1YbA7BzmjwFuwLlvy79EpBrOIIxzVLUJ0BinZzU4vaVj3WU6ikiMRzs7VLU18Avu/UNwhol5\n3mOeFsDfcQZTrA309ixEROoDtwNt3c9OAQbkYt2NyZWgnGcxxnhYDbwmIi8DX6nqL5L9jSS/VNWT\nwEkRmYsTAkuB90UkGPhCVdMCpo+IDMH5/zICZ3iPVe57aXdmXI0zlMhR4Kg4d0ZMGw16iToDPCIi\nnwDt+PP+JwCdgWbAUrfmS3Du42KMT1jAGJMLqrpBRJrhjNP0koh8C5zjz6MBIRkXubAJ/VlEOuDs\n2UwSkVdx9kweA5qr6kERmZChrdPuv6kez9Nep/1/fMFnZXgtOAN9PpXDahqTL+wQmTG54B7iOqGq\nk4HXgKtxRs5t5s5yS4ZFbhKREBEJw7kHzVL3ZlR7VXUc8J7bRihwHDgsIlWB7nkor4U4950PwDkU\nNj/D+z8At4pIFXddKrm1GOMTtgdjTO40Al4VkVSc0YQfwDnU9J6I/ANYnGH+JTj3TKkJ/J+q7hKR\ngcDjInIWOAbcpapbRWQFsBZnmPQFeahtITDCrfFn4HPPN1V1nYj8E/jWDaGzwF+B3/PwWcbkyEZT\nNqYYEJFrgMdUtYe/azEmjR0iM8YY4xO2B2OMMcYnbA/GGGOMT1jAGGOM8QkLGGOMMT5hAWOMMcYn\nLGCMMcb4hAWMMcYYn/j/9RznyOWWEikAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113482320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5.best_score_, gsearch5.best_params_))\n",
    "test_means = gsearch5.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save model to file\n",
    "# xgb5.save_model(dpath+'Model_for_House_Renting.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# save model to file\n",
    "pickle.dump(xgb5, open(\"model.pickle.dat\", \"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "最优结果是Best: -0.580125 using {'colsample_bytree': 0.95, 'subsample': 0.8}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
